# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql
from twisted.enterprise import adbapi

def dbHandle():
    conn = pymysql.connect(
        host='192.168.2.107',
        db='Weather',
        user='root',
        passwd='chiron',
        charset='utf8',
        use_unicode=True
    )
    return conn

class WeatherPipeline(object):
    def __init__(self):
        dbparms = dict(
            host='192.168.2.107',
            db='Weather',
            user='root',
            passwd='chiron',
            charset='utf8',
            cursorclass=pymysql.cursors.DictCursor, # 指定 curosr 类型
            use_unicode=True,
        )
        self.dbpool = adbapi.ConnectionPool("pymysql", **dbparms)

    def process_item(self, item, spider):
        #使用twiest将mysql插入变成异步
        query = self.dbpool.runInteraction(self.do_insert, item)
        #因为异步 可能有些错误不能及时爆出
        query.addErrback(self.handle_error, item, spider) #处理异常
        return item

    #处理异步的异常
    def handle_error(self, failure, item, spider):
        print('failure：',failure)

    def do_insert(self, cursor, item):
        # 查重处理
        cursor.execute("select * from weather where city = %s and date= %s",(item['city'],item['date']))
        # 是否有重复数据
        repetition = cursor.fetchone()
        # 重复
        if repetition:
            pass
        else:
            insert_sql, params = item.get_insert_sql()
            cursor.execute(insert_sql, params)
